import json
import random
import requests
from tools.ua import user_agent_list
from concurrent.futures import ProcessPoolExecutor


class RussiaSpider:
    base_url = 'https://companies.rbc.ru/api/web/v1/search/filter_params/?query='
    category_url = 'https://companies.rbc.ru/api/web/v1/search/filter_params/?category_id={}&query='
    category_time_url = 'https://companies.rbc.ru/api/web/v1/search/filter_params/?category_id={}&query=&registration_date_from={}&registration_date_to={}'
    category_adr_url = 'https://companies.rbc.ru/api/web/v1/search/filter_params/?category_id={}&query=&region_id[]={}&registration_date_from={}&registration_date_to={}'
    category_capital_url = 'https://companies.rbc.ru/api/web/v1/search/filter_params/?capital_from={}&capital_to={}&category_id={}&query=&region_id[]={}&registration_date_from={}&registration_date_to={}'
    final_url = 'https://companies.rbc.ru/api/web/v1/search/filter_params/?capital_from={}&capital_to={}&category_id={}&query=&region_id[]={}&registration_date_from={}&registration_date_to={}&status[]={}'
    un_growth_url = 'https://companies.rbc.ru/api/web/v1/search/filter_params/?capital_from={}&capital_to={}&category_id={}&query=&region_id[]={}&registration_date_from={}&registration_date_to={}&sorting=-revenue_growth_rate&status[]={}'
    growth_url = 'https://companies.rbc.ru/api/web/v1/search/filter_params/?capital_from={}&capital_to={}&category_id={}&query=&region_id[]={}&registration_date_from={}&registration_date_to={}&sorting=revenue_growth_rate&status[]={}'
    detail_time_url = 'https://companies.rbc.ru/search/?category_id={}&query=&registration_date_from={}&registration_date_to={}'
    headers = {
        'user-agent': random.choice(user_agent_list)
    }
    status_code = ['ACTIVE', 'LIQUIDATED', 'LIQUIDATING', 'INVALIDATING', 'REORGANIZING']

    address_code = ['77', '78', '66', '54', '50', '74', '16', '63', '23', '52', '02', '36', '61', '59', '25', '24',
                    '22', '55', '76', '38', '35', '31', '18', '39', '37', '34', '26', '64', '42', '67', '43', '27',
                    '72', '71', '62', '70', '56', '73', '58', '46', '69', '48', '33', '21', '32', '47', '86', '91',
                    '68', '51', '40', '05', '12', '10', '57', '60', '29', '28', '30', '11', '13', '44', '53', '45',
                    '03', '20', '07', '65', '14', '75', '92', '19', '15', '06', '04', '41', '09', '01', '89', '08',
                    '49', '79', '17', '87', '83']
    start_time = 1990
    end_time = 2022
    max_capital = 1000000000
    capital_step = 100000
    is_check_592 = False
    is_check_608 = False

    def handle_req(self):
        '''获取一级分类id'''
        resp = requests.get('https://companies.rbc.ru/api/web/v1/search/filter_params/?query=', headers=self.headers)
        content = resp.content.decode()
        c1_items = json.loads(content)['category']
        c1_ids = [c1['id'] for c1 in c1_items]
        return c1_ids

    def handle_task(self, ids):
        '''使用递归方式获取子分类id'''
        with ProcessPoolExecutor(max_workers=10) as pool:
            for cid in ids:
                if cid == 592:
                    # 判断592是否是重复请求
                    self.handle_592(cid)
                elif cid == 608:
                    self.handle_608(cid)
                else:
                    resp = requests.get(self.category_url.format(cid), headers=self.headers)
                    content = resp.content.decode()
                    tmp_dic = json.loads(content)
                    items = tmp_dic['category']
                    if items:
                        # 还有子分类
                        next_ids = [c2['id'] for c2 in items]
                        pool.submit(self.handle_task, next_ids)
                    else:
                        # 如果下面没有子分类，则直接求值
                        self.handle_detail_by_time(cid)

    def handle_592(self, cid):
        if self.is_check_592:
            print('Repeated requests')
        else:
            self.is_check_592 = True
            resp = requests.get(self.category_url.format(cid), headers=self.headers)
            content = resp.content.decode()
            tmp_dic = json.loads(content)
            items = tmp_dic['category']
            if items:
                # 还有子分类
                next_ids = [c2['id'] for c2 in items]
                self.handle_task(next_ids)

    def handle_608(self, cid):
        if self.is_check_608:
            print('Repeated requests')
        else:
            self.is_check_608 = True
            resp = requests.get(self.category_url.format(cid), headers=self.headers)
            content = resp.content.decode()
            tmp_dic = json.loads(content)
            items = tmp_dic['category']
            if items:
                # 还有子分类
                next_ids = [c2['id'] for c2 in items]
                self.handle_task(next_ids)

    def handle_detail_by_time(self, cid):
        '''根据注册日期给公司分类'''
        for register_time in range(self.start_time, self.end_time):
            time_url = self.category_time_url.format(cid, register_time, register_time + 1)
            resp = requests.get(time_url, headers=self.headers)
            content = resp.content.decode()
            tmp_dic = json.loads(content)
            time_count = tmp_dic['count']
            if time_count > 1000:
                for code in self.address_code:
                    address_count = self.handle_detail_by_address_code(cid, code, register_time)
                    if address_count > 1000:
                        for capital in range(0, self.max_capital, self.capital_step):
                            capital_count = self.handle_detail_by_capital(capital, cid, code, register_time)
                            if capital_count > 1000:
                                for status in self.status_code:
                                    status_count = self.handle_detail_by_status_code(capital, cid, code, register_time,
                                                                                     status)
                                    if status_count > 1000:
                                        self.handle_detail_by_growth(capital, cid, code, register_time,
                                                                                     status)
                            else:
                                print('Normal -.-|||')
                    else:
                        print('Normal -.-|||')
            else:
                # 请求详情页的数据
                print(time_url.replace('https://companies.rbc.ru/api/web/v1/search/filter_params/', ''))
                self.parse_detail(time_url)
            break

    def parse_detail(self, detail_url):
        requests.get(detail_url)

    def handle_detail_by_address_code(self, cid, code, register_time):
        '''根据地址给公司分类'''
        resp = requests.get(
            self.category_adr_url.format(cid, code, register_time,
                                         register_time + 1),
            headers=self.headers)
        content = resp.content.decode()
        tmp_dic = json.loads(content)
        count = tmp_dic['count']
        return count

    def handle_detail_by_capital(self, capital, cid, code, register_time):
        '''根据投资资金给公司分类'''
        resp = requests.get(
            self.category_capital_url.format(capital, capital + self.capital_step, cid, code,
                                             register_time,
                                             register_time + 1),
            headers=self.headers)
        content = resp.content.decode()
        tmp_dic = json.loads(content)
        count = tmp_dic['count']
        return count

    def handle_detail_by_status_code(self, capital, cid, code, register_time, status):
        '''根据公司状态码给公司分类'''
        resp = requests.get(
            self.final_url.format(capital, capital + self.capital_step, cid, code,
                                  register_time,
                                  register_time + 1, status),
            headers=self.headers)
        content = resp.content.decode()
        tmp_dic = json.loads(content)
        count = tmp_dic['count']
        return count

    def handle_detail_by_growth(self, capital, cid, code, register_time, status):
        resp = requests.get(
            self.growth_url.format(capital, capital + self.capital_step, cid, code,
                                  register_time,
                                  register_time + 1, status),
            headers=self.headers)
        content = resp.content.decode()
        tmp_dic = json.loads(content)
        count = tmp_dic['count']

    def run(self):
        # 获取一级分类id
        ids = self.handle_req()
        # 获取二级分类id
        self.handle_task(ids)


if __name__ == '__main__':
    spider = RussiaSpider()
    spider.run()
